#!/usr/bin/env python

# (c) 2007-2011 Helsinki University of Technology and University of Helsinki
# Licensed under the MIT license; see LICENSE.txt for more information.

from worker import Worker
from rdf import *
import sys

class Filter (Worker):
  def __init__(self, session):
    Worker.__init__(self, session.getFatalEvent())
    self._session = session
    
    self._blacklist = set()
    bl = session.getSource().getBlacklist()
    if bl:
      f = open(bl)
      for line in f:
        line = line.strip()
        if line == '' or line.startswith('#'): continue
        self._blacklist.add(line)
      f.close()

  def process(self, doc):
    ident = doc.getIdentifier()
    if str(ident) in self._blacklist:
      doc.log('SKIP', 'filter', 'Blacklisted URI')
      doc.processingFinished()
      return False
    
    noindex = doc.getMetadata(TS.noindex) | doc.getMetadata(TOSU.noindex)
    if noindex:
      val = unicode(list(noindex)[0])
      if val == '1' or val == 'true':
        doc.log('SKIP', 'filter', 'Indexing forbidden using ts:noindex')
        doc.processingFinished()
        return False
  
    doc.setCompleted('filter')
    self._session.schedule(doc)
    return True

  def finish(self):
    # possibly free some memory
    del self._blacklist
